import { PlaywrightCrawler, Dataset, log as logg, playwrightUtils } from 'crawlee';

const crawler = new PlaywrightCrawler({
    launchContext: {
        launchOptions: {
            headless: false,
        },
    },
    maxRequestRetries: 1,
    async requestHandler({ request, page, log }) {
        const { url } = request;
        log.info(`正在处理地址 ${url} 中...`);
        await playwrightUtils.infiniteScroll(page);
        const scrapedData: any = { url, data: [] };
        scrapedData.data = await page.$$eval('#partner_item_box> div > a', ($posts) => {
            const dataArr: any[] = [];
            $posts.forEach(($post: any) => {
                const obj = {
                    title: $post.querySelector('span').innerText,
                    text: $post.querySelector('div').innerText,
                    img: $post.querySelector('img').src,
                    href: $post.href,
                };
                if (JSON.stringify(obj) !== '{}') dataArr.push(obj);
            });
            return dataArr;
        });
        await Dataset.pushData(scrapedData);
    },
});
await crawler.addRequests(['https://nodejs.cn/']);
await crawler.run();
logg.info('爬取并处理完成!');
